package com.coderfans.datagithub;


import com.coderfans.redis.ISetRedis;
import com.coderfans.redis.RedisConfig;
import com.coderfans.redis.impl.RedisCacheClient;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;


/**
 * Created by ${fanchunshuai} on 2016-10-12.
 *
 * @version: V1.0
 * @Desc: 爬取github的所有链接到redis中
 * @Copyright (c) 2016 58到家-版权所有
 */
public class LinkStoreRedis implements PageProcessor {
    //步骤一：抓取网站的相关配置 ： 包括编码 抓取间隔 重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);

    private JedisPool jedisPool = RedisConfig.generatJedisPool();
    private ISetRedis setRedis = new RedisCacheClient(jedisPool);

    @Override
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+)").all());
        System.out.println("url222 = "+page.getUrl().get());


    }

    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new GithubRepoPageProcessor()).addPipeline(new com.coderfans.datagithub.ConsolePipeline())
                .addUrl("https://github.com/lw-lin")
                .thread(8).run();

    }
}
